package fetcher

import (
	"bufio"
	"fmt"
	"golang.org/x/net/html/charset"
	"golang.org/x/text/encoding"
	"golang.org/x/text/encoding/unicode"
	"golang.org/x/text/transform"
	"io/ioutil"
	"learngo/crawler_distribute/config"
	"log"
	"net/http"
	"time"
)

var rateLimiter = time.Tick(time.Second / config.Qps) // 利用定时任务做速率限制 80 * time.Millisecond

func Fetch(url string) ([]byte, error) {
	<-rateLimiter // 限制速率，停止10ms继续执行后续
	//resp, err := http.Get(url) //"http://www.zhenai.com/zhenghun"
	log.Printf("Fetching url %s", url)
	request, err := http.NewRequest(http.MethodGet, url, nil)
	request.Close = true
	request.Header.Add("user-agent",
		"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36")
	client := http.Client{
		CheckRedirect: func(req *http.Request, via []*http.Request) error {
			log.Printf("Redirect: %s", req)
			return nil
		},
	}
	resp, err := client.Do(request)

	defer resp.Body.Close() // 关闭链接

	if err != nil {
		return nil, err
	}

	if resp.StatusCode != http.StatusOK {
		return nil,
			fmt.Errorf("wrong status code: %d", resp.StatusCode)
	}

	// 将判定编码类型的文件读出为utf-8
	bodyReader := bufio.NewReader(resp.Body)
	e := determineEncoding(bodyReader)
	utf8Reader := transform.NewReader(bodyReader, e.NewDecoder())
	return ioutil.ReadAll(utf8Reader)
}

func determineEncoding(r *bufio.Reader) encoding.Encoding {
	// 根据前1024个byte判断编码类型，防止1024字符读完无法再度读取，使用Peek
	bytes, err := r.Peek(1024)
	if err != nil {
		log.Printf("Fetcher error: %v", err)
		return unicode.UTF8
	}
	e, _, _ := charset.DetermineEncoding(bytes, "") //根据前1024个byte判断编码类型
	return e
}
